{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# my_LR_Diabetes糖尿病——Logistic 回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.866045       -0.031990   \n",
       "1  -0.844885                     -1.205066       -0.528319   \n",
       "2   1.233880                      2.016662       -0.693761   \n",
       "3  -0.844885                     -1.073567       -0.528319   \n",
       "4  -1.141852                      0.504422       -2.679076   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.670643      -0.181541  0.166619   \n",
       "1                    -0.012301      -0.181541 -0.852200   \n",
       "2                    -0.012301      -0.181541 -1.332500   \n",
       "3                    -0.695245      -0.540642 -0.633881   \n",
       "4                     0.670643       0.316566  1.549303   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "train = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['Target']   \n",
    "X_train = train.drop([\"Target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "#可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 默认参数的Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [ 0.48797856  0.53011593  0.4562292   0.422546    0.48392885]\n",
      "cv logloss is: <built-in method mean of numpy.ndarray object at 0x0000023DAC830940>\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "#数据集比较大，采用5折交叉验证\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print (('logloss of each fold is: '),-loss)\n",
    "print (('cv logloss is:'), loss.mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression + GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### ogistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） 目标函数为：J = C* sum(logloss(f(xi), yi)) + penalty\n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "\n",
    "    设置参数搜索范围\n",
    "    生成学习器实例（参数设置）\n",
    "    生成GridSearchCV的实例（参数设置）\n",
    "    调用GridSearchCV的fit方法\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=4,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [ 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression(solver='liblinear')\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss',n_jobs = 4,)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.476027893751\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt8VPWd//HXZ2bCRYiCJETk0oCF\neuFmTZVWxEu1UttFEW/bVqtbl3Z/8rP113arbX9WcbfrZd0+dn911wurtZctWq3blFKx2lLErQhS\nFBBRpF6CQhAEjJCQyXx+f5yTMBkmOZMwJxd4Px+PMefyPWc+c0jm7bl9j7k7IiIi7Ul0dwEiItLz\nKSxERCSSwkJERCIpLEREJJLCQkREIiksREQkksJCREQiKSxERCSSwkJERCKluruAYikrK/PKysru\nLkNEpFd5/vnn33X38qh2B01YVFZWsmLFiu4uQ0SkVzGzNwppp8NQIiISSWEhIiKRFBYiIhLpoDln\nISKHrsbGRmpqaqivr+/uUnqsfv36MWLECEpKSjq1vMJCRHq9mpoaSktLqaysxMy6u5wex93Ztm0b\nNTU1jB49ulPr0GEoEen16uvrGTJkiIKiDWbGkCFDDmjPS2EhIgcFBUX7DnT7KCxE5JB06T1/4tJ7\n/tTdZfQaCgsR6VFOeWAWpzwwq7vL6LCBAwe2DE+fPp1Bgwbx2c9+Nm/ba665hsmTJ3P88cfTv39/\nJk+ezOTJk3nkkUc69J4rV67k3ofuZ927Gw6o9kLoBLdIzJq/+JZd9Wg3VyJd5Zvf/Ca7d+/mnnvu\nyTv/rrvuAuD111/ns5/9LKtWrerU+6xcuZKly5dy2iendbrWQmnPQkSkyD75yU9SWlraqWVfffVV\nzj33XE466SSmTZvGK6+8AsD8+fMZP348kyZN4swzz2TPnj3MnTuXBY9WM/OMv+rwXklHac9COuyq\nx68C4IHpD3RzJSL7u/nXa3np7V2R7V56J2hTyHmL448+nO/91QkHXFshZs+ezbx58zjmmGN45pln\nmDNnDk888QQ333wzixcvpqKigh07dtC/f39uvPFGlixfyg3/+F2OK/twrHXFGhZmNh34VyAJzHP3\nW3PmXwncAWwKJ/3Q3eeF8x4HpgBL3T3/gT/pFs1/ZCJSXDt27ODZZ59l1qx952zS6TQAp556Kldc\ncQUXX3wxF154YZfXFltYmFkSuAs4B6gBlptZtbu/lNP0IXefk2cVdwCHAV+Oq0YROfgUugfQvEfx\n0Jc/Hmc5HeLulJWV5T2Hcd9997Fs2TIWLFjApEmTePHFF7u0tjjPWZwMbHD3je6+F5gPnF/owu7+\nFPB+XMWJiPQ0gwcPZtiwYTz22GMAZDIZXnjhBQA2btzIlClTuOWWWxg8eDCbNm2itLSUD+o+6JLa\n4gyL4cBbWeM14bRcs8zsRTN7xMxGxliPiEiXOO2007j44ot56qmnGDFiBIsWLSp42fnz53P33Xcz\nadIkTjjhBBYsWADAddddx4QJE5gwYQJnn30248eP56yzzmL92pe58MwZvfoEd77bBT1n/NfAz929\nwcy+AjwInFXwG5jNBmYDjBo1qrN1iogcsLq6upbhp59+uqBlKisrWbNmTatpY8aMyRsu1dXV+00r\nLy/nF08GeyG9+QR3DZC9pzACeDu7gbtvyxq9D7itI2/g7vcC9wJUVVXlBpGISJt60rmK3iDOw1DL\ngbFmNtrM+gCXAa2i0cyGZY3OANbFWE+bdNu/iEj7YtuzcPe0mc0BFhFcOnu/u681s7nACnevBq41\nsxlAGtgOXNm8vJk9DRwLDDSzGuBL7l74gT8RESmaWO+zcPeFwMKcaTdmDd8A3NDGsqfFWZuIiBRO\nd3ADr/f553BIffeIiOSjvqFE5ND0wGeClxREYSEiUgRd3UX5Y489xh133HHAdRdKh6FERIqsWF2U\np9NpUqn8X9MzZ84E6JJnWYD2LEREiu5AuiifOnUq3/nOd5g2bRo//OEP+dWvfsUpp5zCiSeeyKc+\n9Slqa2sBmDdvHl/72tcA+Pu/+zpf/epX+cQnPsGYMWNaugspJu1ZiMjB5bfXw+bV0e02hx3xFXLe\n4qgJ8Olbo9sVya5du1iyZAkA7733HjNmzMDMuPvuu7nzzju57bb971+ura3lmWeeYfXq1VxyySUt\nex7ForAQEelhLrvsspbhN998k0suuYTNmzfT0NDAuHHj8i5zwQUXYGZMnDiRTZs25W1zIBQWInJw\nKXQPoHmP4qrfxFdLJw0YMKBl+JprruHb3/425513Hk8++SS33pr/8/Xt27dl2L34vR/pnIWISA+2\nc+dOhg8fjrvz4IMPdlsdCgsRkSI7kC7Kc910003MnDmT008/nYqKiiJW2TE6DCUiUgTF6qJ86dKl\nrcZnzZrV6jGrza6++moguHT29v+4s1UX5dm1FIvCQkQOTT3wXEVPpsNQIiISSWEhIiKRFBYiIhJJ\nYSEiIpEUFiJySLrq8au46vGruruMXkNhISJSBM1dlK9atYqPf/zjnHDCCUycOJGHHnpov7bF6KIc\nYOXKlTz91JKi1B9Fl86KiBTRYYcdxo9//GPGjh3L22+/zUknncS5557LoEGDWtoU2kV5lJUrV7J0\n+VJO++S0otTenlj3LMxsupmtN7MNZnZ9nvlXmtlWM1sVvq7OmvdFM3s1fH0xzjpFRIpl3LhxjB07\nFoCjjz6aoUOHsnXr1oKXf/XVVzn33HM56aSTmDZtGq+88goA8+fPZ/z48UyaNIkzzzyTPXv2MHfu\nXBY8Ws3MM/6qw3slHRXbnoWZJYG7gHOAGmC5mVW7+0s5TR9y9zk5yx4JfA+oAhx4Plz2vbjqFZGD\nw23P3cbL21+ObNfcppDzFsceeSzfOvlbHa7lueeeY+/evRxzzDEFLzN79mzmzZvHMcccwzPPPMOc\nOXN44oknuPnmm1m8eDEVFRXs2LGD/v37c+ONN7Jk+VJu+MfvtrqDOw5xHoY6Gdjg7hsBzGw+cD6Q\nGxb5nAv8zt23h8v+DpgO/DymWkVEiuqdd97h8ssv58EHHySRKOwgzo4dO3j22Wdbde+RTqcBOPXU\nU7niiiu4+OKLufDCC2OpuT1xhsVw4K2s8RrglDztZpnZNOAV4Dp3f6uNZYfnLmhms4HZAKNGjSpS\n2SLSmxW6B9C8R/HA9AeKXsOuXbv4zGc+wz/8wz8wZcqUgpdzd8rKyvKew7jvvvtYtmwZCxYsYNKk\nSbz44ovFLDlSnOcsLM+03E7Wfw1UuvtE4Emguf/dQpbF3e919yp3ryovLz+gYkVEimHv3r3MnDmz\nZS+gIwYPHsywYcNaHouayWR44YUXANi4cSNTpkzhlltuYfDgwWzatInS0lI+qPug6J8hnzjDogYY\nmTU+Ang7u4G7b3P3hnD0PuCkQpcVEemJHn74YZYsWcKPfvSjlktiO3K10/z587n77ruZNGkSJ5xw\nAgsWLADguuuuY8KECUyYMIGzzz6b8ePHc9ZZZ7F+7ctceOaM3nuCG1gOjDWz0cAm4DLgc9kNzGyY\nu78Tjs4A1oXDi4Dvm9ngcPxTwA0x1ioickCauwX/whe+wBe+8IWClsnXRfmYMWPyPv+iurp6v2nl\n5eX84slgL6TXnuB297SZzSH44k8C97v7WjObC6xw92rgWjObAaSB7cCV4bLbzewWgsABmNt8sltE\npBjiOFdxMIv1pjx3XwgszJl2Y9bwDbSxx+Du9wP3x1mfiIgURt19iIhIJIWFiIhEUliIiEgkhYWI\nHJLeuPwK3rj8iu4uo9dQWIiIFEFXd1H+2GOPcccddxSt/ijqolxEpIiK2UV5Op0mlcr/NT1z5kwA\n1r27ocifID+FhYhIEY0bN65lOLuL8uywaM/UqVM5/fTTefrpp7nwwgsZPXo03//+99m7dy/l5eX8\n9Kc/ZejQocybN481a9bw5e/O4e//7uuMObqS5cuXs3nzZu68886WMCkWhYWIHFQ2f//7NKyL7qK8\n/uWgTSHnLfoedyxHffvbHa6lM12UQ9AR4ZIlwRPw3nvvPWbMmIGZcffdd3PnnXdy22237bdMbW0t\nzzzzDKtXr+aSSy5RWIiI9Aad6aK82WWXXdYy/Oabb3LJJZewefNmGhoaWu25ZLvgggswMyZOnMim\nTZsOqPZ8FBYiclApdA+geY/iQz/5cdFr6GwX5c0GDBjQMnzNNdfw7W9/m/POO48nn3ySW2+9Ne8y\nffv2bRl236+T7gOmq6FERIroQLooz2fnzp0MHz4cd+fBBx+MXiAmCgsRkSI60C7Kc910003MnDmT\n008/nYqKiiJW2jE6DCUiUgTF6qJ86dKlrcZnzZrV6jGrza6++moguHT29v+4s1UX5c21FJPCQkQO\nSXGcqziY6TCUiIhEUliIyEEhjiuADiYHun0UFiLS6/Xr149t27YpMNrg7mzbto1+/fp1eh06ZyEi\nvd6IESOoqalh69at3V1Kl3unLvzMWxvbbdevXz9GjBjR6feJNSzMbDrwrwTP4J7n7nnvJjGzi4Bf\nAB9z9xVm1ge4B6gCMsBX3X1xnLWKSO9VUlLC6NGju7uMbnHlA98FYNlVj8b6PrGFhZklgbuAc4Aa\nYLmZVbv7SzntSoFrgWVZk/8WwN0nmNlQ4Ldm9jF3z8RVr4iItC3OcxYnAxvcfaO77wXmA+fnaXcL\ncDtQnzXteOApAHevBXYQ7GWIiEg3iDMshgNvZY3XhNNamNmJwEh3X5Cz7AvA+WaWMrPRwEnAyNw3\nMLPZZrbCzFYciscqRUS6SpznLCzPtJZLFcwsAfwAuDJPu/uB44AVwBvA/wDp/Vbmfi9wL0BVVZUu\ngxARiUmcYVFD672BEcDbWeOlwHhgsZkBHAVUm9kMd18BXNfc0Mz+B3g1xlpFRKQdcR6GWg6MNbPR\n4dVNlwHVzTPdfae7l7l7pbtXAs8CM8KroQ4zswEAZnYOkM49MS4iIl0ntj0Ld0+b2RxgEcGls/e7\n+1ozmwuscPfqdhYfCiwyswywCbg8rjpFRCRarPdZuPtCYGHOtBvbaHtG1vDrwEfirE1ERAqn7j5E\nRCSSwkJERCIpLEREJJLCQkREIiksREQkksJCREQiKSxERCSSwkJERCIpLEREJJLCQkREIiksREQk\nksJCREQiKSxERCSSwkJERCIpLEREJJLCQkREInU4LMwsYWaHx1GMiIj0TAWFhZn9l5kdHj4X+yVg\nvZl9M97SRESkpyh0z+J4d98FXEDwmNRRFPBcbDObbmbrzWyDmV3fTruLzMzNrCocLzGzB81stZmt\nM7MbCqxTRERiUGhYlJhZCUFY/MrdGwFvbwEzSwJ3AZ8Gjgf+2syOz9OuFLgWWJY1+WKgr7tPAE4C\nvmxmlQXWKiIiRVZoWNwDvA4MAJaY2YeAXRHLnAxscPeN7r4XmA+cn6fdLcDtQH3WNAcGmFkK6A/s\nLeD9REQkJgWFhbv/m7sPd/fzPPAGcGbEYsOBt7LGa8JpLczsRGCkuy/IWfYR4APgHeBN4J/dfXsh\ntYqISPEVeoL7q+EJbjOz/zSzlcBZUYvlmdZy6MrMEsAPgK/naXcy0AQcDYwGvm5mY/LUNdvMVpjZ\niq1btxbyUUREpBMKPQz1N+EJ7k8B5cBVwK0Ry9QAI7PGRwBvZ42XAuOBxWb2OjAFqA5Pcn8OeNzd\nG929FngGqMp9A3e/192r3L2qvLy8wI8iIiIdVWhYNO8lnAc84O4vkH/PIdtyYKyZjTazPsBlQHXz\nTHff6e5l7l7p7pXAs8AMd19BcOjprHBPZgBBkLxc8KfqgEzGadxTRlPjAOobm+J4CxGRXi9VYLvn\nzewJgkNCN4RXMGXaW8Dd02Y2B1gEJIH73X2tmc0FVrh7dTuL3wU8AKwhCKUH3P3FAmvtkPd272X7\nmzMAOPb/Pk5pvxTlA/tSNrAvZaV9KBvYNxgvDacNDKeV9qVfSTKOkkREepxCw+JLwGRgo7vvNrMh\nBIei2uXuCwnuy8iedmMbbc/IGq4juHw2dgP6phg0/Hdk0v348uSr2fp+A1vrGnj3/QbWb36fZ+q2\nsXNPY95lS/umwhDZFyBlA7NCpTQIGgWLiPR2BYWFu2fMbATwOTMD+KO7/zrWyrpIv5Ik4/ouhr5w\nzZn5T8M0pJvYVreXd+sagtf7e9la18DW9xtapr1aW8f/vNZ2sAzsm8ofKqV9WvZcmvdo+vdRsIhI\nz1JQWJjZrcDHgJ+Fk641s0+4+yFxZ3XfVJKjB/Xn6EH9I9vuTWfY9kEQKO+GgbK1OWTq9vLu+0Gw\n/GnjNnbszh8sA/ok9wuU3JApD6cf1qfQnUMRkc4r9JvmPGCyu2cAzOxB4M9Arw+L+nQ925MZzOHR\nVx6lJFlCylKkEilKEiXBz3BaSTIYbx4usZKWac1tU4kUFYf3ZdgRhQXL9g/yhEq45/Lu+w28trWO\nZX9p4L12giXf+ZSWUMkKGgWLiHRWR749BgHNN8YdEUMt3aKusY4tqeD2j5v+dFPR1puwxL6wyQqS\n3Gl555WW0P+IFGMSJXwknJawFOm00Zg26huNhkajfi/s3uvsaYAPGpwtu5y6WqeuHvAE7knwJJDE\nPUm/VAmD+vfnyP79GHxYf8oGHkbZgMMoLz2MoQP7U1E6gIrDB1BRehgD+nZNsLg7Gc8ELzI0ZZpa\nhjOZ8KfvezV5U6vx3HnunrdNkzeRzmRobEqHP5tIZ5pozDTRlGlqGU9nMuHP4NXUMh7UlvZ90zKe\nCaaH05oyTTR5hiYPam/yYPyDhgw4XPLIN8PP3GoLtPrvvnnZ08HxlhHfr6cdz1pv67ltra9VO/c2\nWuSuO0+rPDXlVJDTtr33CtTVB/9jdOaP/4Z4tdtjUez2/3fsnLr6RvB+RVlXewr9Rvgn4M9m9geC\nq5OmcRDsVQAc2e9IxjUkcODfP7+IxkwjjZlG0pl0y8/s4camRtK+72e+aY1N4Tqap2Uvn2+dmUYa\n0g3UZeryzstdLu3p/T9IChgYvNrbp3k/fL0BUBe+trRu427gSYwkCUuRtCRJK6EkkaIkkWK37wSc\nsx/6LBlvIuNOxptwmr/4m8jguGfw8Mveg7l42M6DFnT3H2wxuRuQADfAwp8J6GO4G2t3Zt9mtP+V\n59bG9PaWyT/P2piTu3xb68s/ff/62q7HCmzXumU4lGoEjPca97SzXLFE3QHQC94l2Yg3xX+es9AT\n3D83s8UE5y0M+Ja7b46zsK6SsATJ8J/yqAFHdXM1hcl48H+x+YItKpRy59WnG9hV38COPXvY1dDA\nrvp66hoaqGvYyweN9eze28iexgbq042837QXLEOwkwk1HyTY96Vo4MG4Zw3v+2mYWRhACRIkMEuQ\nMCNh+6YlLOuVSJCwJMlwfN/PJMlE8DORSJAK26YsSTIRtE8mkqQsQTIZ/kwkSSUSpBJJUolg+ZJw\nuCQZjieT4R5eIjz8GLQvSSRJJYN2JckkfcLxPskUqUSSPslg3SWpBMmEkUo0/zSSCWPaTy4Fc569\n8hEs/NYw65ovqd7olAdmAbDsqke7uZLe4ZQHZkFJ/O/TbliY2UdzJtWEP482s6PdfWU8ZUl7EpYg\nkUxQkuyC35As6aYM23fvZfpP/zdg/OyCf275QixJtv6CTCUSJJP7xpNmJBKH5hekWbD3dKh+fjk4\nRO1Z3NnOPCe6fyg5iKSSCYaW9qOkX3Dq6rhhemCiyKGi3bBw96ieZUVE5BBQ6H0WF+aZvBNYHXb0\nJyIiB7GOdPfxceAP4fgZBB3/jTOzue7+kxhqExGRHqLQsMgAx7n7FgAzqwD+AzgFWAIoLEREDmKF\nhkVlc1CEaoFx7r7dzPLfWtyLXPnw3mBgdvfWISLSUxUaFk+b2QLgF+H4RQTP4h4A7IilMhER6TEK\nDYtrgAuBqQQ35T0IPOruTvSzuEVEpJcr9A5uN7OlwF6C+yueC4NCREQOAQU9VtXMLgGeIzj8dAmw\nzMwuirMwERHpOQo9DPUd4GPN91SYWTnwJPBIXIWJiEjPUdCeBZDIufluWyHLmtl0M1tvZhvM7Pp2\n2l1kZm5mVeH4581sVdYrY2aTC6xVRESKrNA9i8fNbBHw83D8UnKerZ3LzJLAXcA5BB0QLjezand/\nKaddKXAtsKx5mrv/jPCpfGY2AfiVu68qsFaJ2fd+ti4YiHwKu4C2V0dpe3VMV22vgvYs3P2bwL3A\nRGAScK+7fytisZOBDe6+0d33AvOB8/O0uwW4HahvYz1/zb6QEhGRblDw49Dc/VGgIx3MDwfeyhqv\nIbjju4WZnQiMdPcFZvaNNtZzKflDpjgyTUwcu50d7/eBreuhbBzoWQMiIq1EPc/iffI/yswIrqht\nr4/qfN+4LesyswTwA+DKdt7/FGC3u69pY/5swvuuR40a1U4p7dj1NkMGNTCiYg/cdTIMKIfKqVB5\nWvAqG6vwEJFDXlQX5aUHsO4aYGTW+Agg+7mSpcB4YHH41LCjgGozm+HuK8I2l9HOISh3v5fg8BhV\nVVWdu+9j0Ej+sPwoDuvXxBk3fwdeXwqvPw1rHwvmDxgahMfoMDyGfFjhISKHnIIPQ3XCcmCsmY0G\nNhF88X+ueaa77wTKmsfDx7Z+ozkowj2Piwme9x0zY3d9Ck76YvByh+0bg9B4fSn85WlY+8ug6cCK\ncM9jKlROgyHHKDxE5KAXW1i4e9rM5gCLgCRwv7uvNbO5wAp3r45YxTSgxt03xlVjm8yCEBhyDJx0\nZevw+EsYIGvC0zcDj8oKj9MUHiJyUIpzzwJ3X0jOJbbufmMbbc/IGV8MTImrtg7JFx7bXtu35/H6\n07AmvD+xdFjr8DhyjMJDRHq9WMPioGUGZR8OXlVXheGxYV94bPwjrA476C09Ois8pio8RKRXUlgU\ng1lw1VTZWKj6myA83n01KzwWw+qHg7aHD28dHoNHKzxEpMdTWMTBDMrHBa+PfSkrPJYE4fHa7+HF\nh4K2LeFxWhgelQoPEelxFBZdoVV4XB2Gxyv7TphveCorPEZkXao7FQZ9SOEhIt1OYdEdzKD8I8Gr\nOTy2rt932GrDk/Di/KDtESNbnzAf/KHurV1EDkmHfFi4O6m0k052YxFmMPTY4HXy34bh8fK+K61e\nfQJeCO9NPGLUvvAYfRoM6uSd6yIiHXDIh0XTjh2M3BwMvzLl46SGDiVVUUFqaDmpoUMpqagIppUP\nDX6WDcGSMSeLGQw9Lnid/LeQybQOj1cehxf+K2g7aNS+8x2VUxUeIhKLQz4sEn36sHUwpJpg9Ken\nk95SS7q2lob160m/+27wRd1qgQSpsrJWoVIydCipoWGoDB1Kamg5yUGDsGKda0gkoOL44HXK7Kzw\neDp4rf8trPpZ0HbQh3LCY2T76xYRKYDCYsAA6gYEX+rDvve9VvM8nSa9bTvp2lrStVtI19bSWBuE\nSXpLLY1vvcWe55+naceO/dZrffrsC4+KoWGgNL/2hUxiwIBOFJ0dHl8Ow2Nd2DXJElj/G1j106Dt\n4MrWV1sdMaLj7ycih7xDPizaY6kUJRVDKakYStDnYX6ZhgbSW7eGoVJLesuWMFS2kt6yhYZ1L1P3\nxyX47t37LZsYOLDVHklJRcW+Q17NIVNejvXp03ahiQRUnBC8msOj9qV9h63WLYA/Z4fHaVnhMfzA\nNpKIHBIUFkWQ6NuXPiNG0GdE+//X3lT3QcseSrq2lsYtW4JACcf3rHieXVu3QmPjfssmBw9ufS6l\n1WGvINCSRx4ZnE9JJOCo8cFrylfC8FgbhsdSWPdr+PNPghUPHh2eLJ8W/Dz86Dg2kYj0cgqLLpQc\nOIDkwDH0HTOmzTbuTtOOHaS3ZIVKeNirebx+3Tqa3t0WXDXV6g2SwfmUNs+lfJKS4z5HYuBArFV4\nVO8LjyPHtD5spfAQERQWPY6ZkRo8mNTgwXDssW22C86nbMs57LUvVBrfeJPdy1eQ2blz//fo2zfr\nMNdRpMr/jtSgDCmvJfXBRkqerSb13E9IpByOPCYnPIbF+fFFpIdSWPRSwfmUCkoqKmDChDbbZerr\n951PyTmXkq6tpX7tSzTW1uJ79mQtdRhwGIn+fUgNyJAqeZySvr8i1b+JVNmRHHtEHY0Zo+7/XRMW\nY4CBJYLnIzaPQ3BIDAumhdNbrhLbb17uulrPs+Zp7FsXCdtv/bnzLHtdZM2z5vcnZ137t7fmeS01\nJ/YtZ3mWaf6siQRHJBvBoeG533XiX/rQc0SiETdoWP5kd5fSKxyebCTj8ffyoLA4yCX69aPPyJH0\nGdn2JbTuTqaurs1zKektW/jgnU2k39oO6xqB4Aqut9b9vos+Re92ZPhntvGKa7u5kt6hZXtd/r+7\nuZLeYQgpSgbtf56z2BQWgpmRLC0lWVpK32OOabOdZzI0vfcei2dOBeDUHz4UnDx3gAx4Bs844OH5\nlPBnJpM1LWzvmWA8q523Gs9A87qal/Vwess4wXLN62x+n1btw/fcr64gJIN6fL9agvch7zz3nPcg\n6/OQ2x7+8siPABg964oi/Gsd/P7y6I8BGH3h5d1cSe/wl1/+hA+64H0UFlIwSyRIDRnC3j7BLm//\niRO7uaLeYdNDQVctk2Z/L6KlAGx6OOhUc9KXb+reQnqJTb94uEveJ9El7yIiIr1arGFhZtPNbL2Z\nbTCz69tpd5GZuZlVZU2baGZ/MrO1ZrbazPrFVWeD9afB+se1ehGRXi+2w1BmlgTuAs4BaoDlZlbt\n7i/ltCsFrgWWZU1LAT8FLnf3F8xsCBD/GRwREckrzj2Lk4EN7r7R3fcC84Hz87S7BbgdqM+a9ing\nRXd/AcDdt7l7U4y1iohIO+IMi+HAW1njNeG0FmZ2IjDS3RfkLDsOcDNbZGYrzezvY6xTREQixHk1\nVL67RFr6pzCzBPAD4Mo87VLAVOBjwG7gKTN73t2favUGZrOB2QCjRuk5DiIicYlzz6IGyL4TbATw\ndtZ4KUFXrovN7HVgClAdnuSuAf7o7u+6+25gIfDR3Ddw93vdvcrdq8rLy2P6GCIiEmdYLAfGmtlo\nM+sDXAZUN890953uXubule5eCTwLzHD3FcAiYKKZHRae7D4deGn/txARka4QW1i4exqYQ/DFvw54\n2N3XmtlcM5sRsex7wL8QBM4wBqDbAAAK1ElEQVQqYKW7/yauWkVEpH2x3sHt7gsJDiFlT7uxjbZn\n5Iz/lODyWRER6Wa6g1tERCKpbyjg1s9VAjCze8sQEemxtGchIiKRFBYiIhJJYSEiIpEUFiIiEklh\nISIikRQWIiISSWEhIiKRFBYiIhJJN+VJh938+eMAOK+b6+gttL06RturY7pqe2nPQkREIiksREQk\nksJCREQiKSxERCSSwkJERCIpLEREJJLCQkREIiksREQkUqxhYWbTzWy9mW0ws+vbaXeRmbmZVYXj\nlWa2x8xWha+746xTRETaF9sd3GaWBO4CzgFqgOVmVu3uL+W0KwWuBZblrOI1d58cV30iIlK4OPcs\nTgY2uPtGd98LzAfOz9PuFuB2oD7GWkRE5ADEGRbDgbeyxmvCaS3M7ERgpLsvyLP8aDP7s5n90cxO\ny/cGZjbbzFaY2YqtW7cWrXAREWktzrCwPNO8ZaZZAvgB8PU87d4BRrn7icD/Af7LzA7fb2Xu97p7\nlbtXlZeXF6lsERHJFWdY1AAjs8ZHAG9njZcC44HFZvY6MAWoNrMqd29w920A7v488BowLsZaRUSk\nHXGGxXJgrJmNNrM+wGVAdfNMd9/p7mXuXunulcCzwAx3X2Fm5eEJcsxsDDAW2BhjrSIi0o7YroZy\n97SZzQEWAUngfndfa2ZzgRXuXt3O4tOAuWaWBpqAr7j79rhqFRGR9sX68CN3XwgszJl2Yxttz8ga\nfhR4NM7aRESkcLqDW0REIumxqsDxw/a70EpERLJoz0JERCIpLEREJJLCQkREIiksREQkksJCREQi\n6Woo4IHpD3R3CSIiPZr2LEREJJLCQkREIiksREQkksJCREQiKSxERCSSwkJERCIpLEREJJLCQkRE\nIiksREQkksJCREQixRoWZjbdzNab2QYzu76ddheZmZtZVc70UWZWZ2bfiLNOERFpX2xhYWZJ4C7g\n08DxwF+b2fF52pUC1wLL8qzmB8Bv46pRREQKE2dHgicDG9x9I4CZzQfOB17KaXcLcDvQau/BzC4A\nNgIfxFijdMKyqx7t7hJEpIvFGRbDgbeyxmuAU7IbmNmJwEh3X5B9qMnMBgDfAs4hJ0Rylp8NzAYY\nNWpU8SoXKSKFa8doe3VMV22vOM9ZWJ5p3jLTLEFwmOnredrdDPzA3evaewN3v9fdq9y9qry8/ICK\nFRGRtsW5Z1EDjMwaHwG8nTVeCowHFpsZwFFAtZnNINgDucjMbgcGARkzq3f3H8ZYr4iItCHOsFgO\njDWz0cAm4DLgc80z3X0nUNY8bmaLgW+4+wrgtKzpNwF1CgoRke4T22Eod08Dc4BFwDrgYXdfa2Zz\nw70HERHpJczdo1v1AlVVVb5ixYruLkNEpFcxs+fdvSqqne7gFhGRSAoLERGJpLAQEZFICgsREYl0\n0JzgNrOtwBsHsIoy4N0ilVNMqqtjVFfHqK6OORjr+pC7R97VfNCExYEysxWFXBHQ1VRXx6iujlFd\nHXMo16XDUCIiEklhISIikRQW+9zb3QW0QXV1jOrqGNXVMYdsXTpnISIikbRnISIikQ7ZsDCzi81s\nrZllcp/9ndOuoOeIF7GuI83sd2b2avhzcBvtmsxsVfiqjrGedj+/mfU1s4fC+cvMrDKuWjpQ05Vm\ntjVr+1wdd03h+95vZrVmtqaN+WZm/xbW/aKZfbSH1HWGme3M2l43dlFdI83sD2a2Lvxb/GqeNl2+\nzQqsq8u3mZn1M7PnzOyFsK6b87SJ7+/R3Q/JF3Ac8BFgMVDVRpsk8BowBugDvAAcH3NdtwPXh8PX\nA7e10a6uC7ZR5OcH/hdwdzh8GfBQD6jpSuCH3fA7NQ34KLCmjfnnETxT3oApwLIeUtcZwIJu2F7D\ngI+Gw6XAK3n+Lbt8mxVYV5dvs3AbDAyHS4BlwJScNrH9PR6yexbuvs7d10c0a3mOuLvvBZqfIx6n\n84EHw+EHgQtifr/2FPL5s+t9BPikhU+z6saauoW7LwG2t9PkfODHHngWGGRmw3pAXd3C3d9x95Xh\n8PsEjzIYntOsy7dZgXV1uXAbND89tCR85Z50ju3v8ZANiwLle4543L80Fe7+DgS/tMDQNtr1M7MV\nZvasmcUVKIV8/pY2HjzDZCcwJKZ6Cq0JYFZ42OIRMxuZZ3536I7fp0J9PDy88VszO6Gr3zw8XHIi\nwf8tZ+vWbdZOXdAN28zMkma2CqgFfufubW6vYv89xvmkvG5nZk8SPK4113fc/VeFrCLPtAO+fKy9\nujqwmlHu/raZjQF+b2ar3f21A60tRyGfP5Zt1I5C3u/XwM/dvcHMvkLwf1pnxVhTobp6WxVqJUGX\nD3Vmdh7w38DYrnpzMxsIPAp8zd135c7Os0iXbLOIurplm7l7EzDZzAYBj5nZeHfPPhcV2/Y6qMPC\n3c8+wFVEPUe8U9qry8y2mNkwd38n3N2ubWMdb4c/N1rwSNoTCY7lF1Mhn7+5TY2ZpYAjiPeQR2RN\n7r4ta/Q+4LYY6+mIWH6fDlT2F6G7LzSzfzezMnePvQ8kMysh+EL+mbv/Mk+TbtlmUXV15zYL33NH\n+Hc/HcgOi9j+HnUYqn0tzxE3sz4EJ4xiu/IoVA18MRz+IrDfHpCZDTazvuFwGXAq8FIMtRTy+bPr\nvQj4vYdn12ISWVPOMe0ZBMece4Jq4IrwCp8pwM7mQ47dycyOaj6ubWYnE3wvbGt/qaK8rwH/Caxz\n939po1mXb7NC6uqObWZm5eEeBWbWHzgbeDmnWXx/j115Nr8nvYCZBCncAGwBFoXTjwYWZrU7j+Bq\niNcIDl/FXdcQ4Cng1fDnkeH0KmBeOPwJYDXBlUCrgS/FWM9+nx+YC8wIh/sBvwA2AM8BY7pgG0XV\n9E/A2nD7/AE4tot+p34OvAM0hr9bXwK+AnwlnG/AXWHdq2njKrxuqGtO1vZ6FvhEF9U1leAQyYvA\nqvB1XndvswLr6vJtBkwE/hzWtQa4MZzeJX+PuoNbREQi6TCUiIhEUliIiEgkhYWIiERSWIiISCSF\nhYiIRFJYiHSAmdVFt2p3+UfCu+4xs4Fmdo+ZvRb2IrrEzE4xsz7h8EF906z0LgoLkS4S9h+UdPeN\n4aR5BHfXjnX3Ewh6yy3zoIPEp4BLu6VQkTwUFiKdEN5RfIeZrTGz1WZ2aTg9EXb9sNbMFpjZQjO7\nKFzs84R35JvZMcApwHfdPQNB1y3u/puw7X+H7UV6BO3minTOhcBkYBJQBiw3syUEXa9UAhMIegxe\nB9wfLnMqwd3UACcAqzzoGC6fNcDHYqlcpBO0ZyHSOVMJerZtcvctwB8JvtynAr9w94y7bybobqTZ\nMGBrISsPQ2SvmZUWuW6RTlFYiHROWw+Uae9BM3sI+u6BoF+hSWbW3t9gX6C+E7WJFJ3CQqRzlgCX\nhg+jKSd4dOlzwFKCBy8lzKyC4PGbzdYBHwbw4NkjK4Cbs3ovHWtm54fDQ4Ct7t7YVR9IpD0KC5HO\neYyg988XgN8Dfx8ednqUoGfXNcA9BE9Y2xku8xtah8fVBA/B2mBmqwmevdH8rIYzgYXxfgSRwqnX\nWZEiM7OBHjxBbQjB3sap7r45fAbBH8Lxtk5sN6/jl8ANHv2ceJEuoauhRIpvQfiQmj7ALeEeB+6+\nx8y+R/Cc5DfbWjh8qNN/KyikJ9GehYiIRNI5CxERiaSwEBGRSAoLERGJpLAQEZFICgsREYmksBAR\nkUj/Hzglj2QGpaqVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x23dac1dc5c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)=77777777777777\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的logloss。 可以看出在训练集和测试集上都是C越大（正则越少）的模型性能越好"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
